{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 加载库"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\program\\anaconda\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "D:\\program\\anaconda\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "D:\\program\\anaconda\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "D:\\program\\anaconda\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "D:\\program\\anaconda\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "D:\\program\\anaconda\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "D:\\program\\anaconda\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "D:\\program\\anaconda\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "D:\\program\\anaconda\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "D:\\program\\anaconda\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "D:\\program\\anaconda\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "D:\\program\\anaconda\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import csv\n",
    "import time\n",
    "import datetime\n",
    "import random\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score\n",
    "from tools.metrics import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 配置"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from config import Config\n",
    "config = Config()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据加载"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "code_folding": [
     5,
     9,
     37
    ],
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "if config.wordvecType==\"fasttext\":\n",
    "    from tools.utils_fasttext import Dataset,nextBatch\n",
    "else:\n",
    "    from tools.utils import Dataset,nextBatch\n",
    "    \n",
    "if config.layerType==\"AdversarialLSTM\":\n",
    "    from collections import Counter\n",
    "    class ADataset(Dataset):\n",
    "        def __init__(self,config):  # 先继承，在重构\n",
    "            Dataset.__init__(self,config)  #继承父类的构造方法，也可以写成：super(Chinese,self).__init__(name,age)\n",
    "            self.indexFreqs = []  # 统计词空间中的词在出现在多少个review中\n",
    "        # 生成词向量和词汇-索引映射字典，可以用全数据集\n",
    "        def _genVocabulary(self, reviews, labels):\n",
    "            \"\"\"\n",
    "            生成词向量和词汇-索引映射字典，可以用全数据集\n",
    "            \"\"\"\n",
    "            allWords = [word for review in reviews for word in review]\n",
    "            if self.isCleanStopWord:\n",
    "                # 去掉停用词\n",
    "                subWords = [word for word in allWords if word not in self.stopWordDict]\n",
    "                wordCount = Counter(subWords)  # 统计词频\n",
    "            else:\n",
    "                wordCount = Counter(allWords)  # 统计词频\n",
    "            sortWordCount = sorted(wordCount.items(), key=lambda x: x[1], reverse=True)\n",
    "            # 去除低频词\n",
    "            words = [item[0] for item in sortWordCount if item[1] >= 5]\n",
    "            vocab, wordEmbedding = self._getWordEmbedding(words)\n",
    "            self.wordEmbedding = wordEmbedding\n",
    "            word2idx = dict(zip(vocab, list(range(len(vocab)))))\n",
    "            uniqueLabel = list(set(labels))\n",
    "            label2idx = dict(zip(uniqueLabel, list(range(len(uniqueLabel)))))\n",
    "            # 得到逆词频\n",
    "            self._getWordIndexFreq(vocab, reviews, word2idx)\n",
    "            self.labelList = list(range(len(uniqueLabel)))\n",
    "            # 将词汇-索引映射表保存为json数据，之后做inference时直接加载来处理数据\n",
    "            with open(self.word2idxSource, \"w\", encoding=\"utf-8\") as f:\n",
    "                json.dump(word2idx, f)\n",
    "            with open(self.label2idxSource, \"w\", encoding=\"utf-8\") as f:\n",
    "                json.dump(label2idx, f)\n",
    "            return word2idx, label2idx \n",
    "        def _getWordIndexFreq(self, vocab, reviews, word2idx):\n",
    "            \"\"\"\n",
    "            统计词汇空间中各个词出现在多少个文本中\n",
    "            \"\"\"\n",
    "            reviewDicts = [dict(zip(review, range(len(review)))) for review in reviews]\n",
    "            indexFreqs = [0] * len(vocab)\n",
    "            for word in vocab:\n",
    "                count = 0\n",
    "                for review in reviewDicts:\n",
    "                    if word in review:\n",
    "                        count += 1\n",
    "                indexFreqs[word2idx[word]] = count\n",
    "            self.indexFreqs = indexFreqs\n",
    "    data = ADataset(config)\n",
    "    data.dataGen()\n",
    "else:\n",
    "    data = Dataset(config)\n",
    "    data.dataGen()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 模型训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 学习率衰减方法：https://www.jianshu.com/p/c55e8606d70b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From D:\\project\\python_wp\\nlp\\TextClassifier\\layers\\textcnn.py:15: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "WARNING:tensorflow:From D:\\project\\python_wp\\nlp\\TextClassifier\\layers\\textcnn.py:37: The name tf.truncated_normal is deprecated. Please use tf.random.truncated_normal instead.\n",
      "\n",
      "WARNING:tensorflow:From D:\\project\\python_wp\\nlp\\TextClassifier\\layers\\textcnn.py:49: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
      "\n",
      "WARNING:tensorflow:From D:\\project\\python_wp\\nlp\\TextClassifier\\layers\\textcnn.py:65: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From D:\\project\\python_wp\\nlp\\TextClassifier\\layers\\textcnn.py:68: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
      "\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From D:\\project\\python_wp\\nlp\\TextClassifier\\layers\\textcnn.py:73: The name tf.nn.xw_plus_b is deprecated. Please use tf.compat.v1.nn.xw_plus_b instead.\n",
      "\n",
      "WARNING:tensorflow:From D:\\program\\anaconda\\lib\\site-packages\\tensorflow\\python\\ops\\clip_ops.py:286: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "Writing to D:\\project\\python_wp\\nlp\\TextClassifier\\summarys\\textCNN\n",
      "\n",
      "start training model\n",
      "train: step: 1, loss: 5.254478931427002\n",
      "train: step: 2, loss: 4.708699703216553\n",
      "train: step: 3, loss: 5.004139423370361\n",
      "train: step: 4, loss: 3.9531521797180176\n",
      "train: step: 5, loss: 3.8539607524871826\n",
      "train: step: 6, loss: 3.6608328819274902\n",
      "train: step: 7, loss: 3.7710349559783936\n",
      "train: step: 8, loss: 3.4990954399108887\n",
      "train: step: 9, loss: 3.19201397895813\n",
      "train: step: 10, loss: 3.6733040809631348\n",
      "train: step: 11, loss: 4.038313388824463\n",
      "train: step: 12, loss: 3.8014888763427734\n",
      "start training model\n",
      "train: step: 13, loss: 3.2297773361206055\n",
      "train: step: 14, loss: 2.9920270442962646\n",
      "train: step: 15, loss: 3.30033802986145\n",
      "train: step: 16, loss: 2.8625876903533936\n",
      "train: step: 17, loss: 2.554537296295166\n",
      "train: step: 18, loss: 2.700366258621216\n",
      "train: step: 19, loss: 3.0440833568573\n",
      "train: step: 20, loss: 2.9022772312164307\n",
      "train: step: 21, loss: 2.77707576751709\n",
      "train: step: 22, loss: 2.919429063796997\n",
      "train: step: 23, loss: 2.7571990489959717\n",
      "train: step: 24, loss: 2.6064443588256836\n",
      "start training model\n",
      "train: step: 25, loss: 3.0003368854522705\n",
      "train: step: 26, loss: 2.7286722660064697\n",
      "train: step: 27, loss: 2.155529022216797\n",
      "train: step: 28, loss: 2.183544635772705\n",
      "train: step: 29, loss: 2.7044126987457275\n",
      "train: step: 30, loss: 2.3237714767456055\n",
      "train: step: 31, loss: 2.494551420211792\n",
      "train: step: 32, loss: 2.4497222900390625\n",
      "train: step: 33, loss: 2.817110300064087\n",
      "train: step: 34, loss: 2.340850591659546\n",
      "train: step: 35, loss: 2.8612523078918457\n",
      "train: step: 36, loss: 2.2166078090667725\n",
      "start training model\n",
      "train: step: 37, loss: 2.098602771759033\n",
      "train: step: 38, loss: 2.5042901039123535\n",
      "train: step: 39, loss: 2.234093427658081\n",
      "train: step: 40, loss: 2.2017507553100586\n",
      "train: step: 41, loss: 1.9974521398544312\n",
      "train: step: 42, loss: 1.8647645711898804\n",
      "train: step: 43, loss: 2.2378056049346924\n",
      "train: step: 44, loss: 2.3063831329345703\n",
      "train: step: 45, loss: 2.094550371170044\n",
      "train: step: 46, loss: 1.653558611869812\n",
      "train: step: 47, loss: 2.0106964111328125\n",
      "train: step: 48, loss: 2.1004819869995117\n",
      "start training model\n",
      "train: step: 49, loss: 2.050699234008789\n",
      "train: step: 50, loss: 2.2325828075408936\n",
      "train: step: 51, loss: 1.8742436170578003\n",
      "train: step: 52, loss: 1.720553994178772\n",
      "train: step: 53, loss: 1.8203400373458862\n",
      "train: step: 54, loss: 2.0671639442443848\n",
      "train: step: 55, loss: 2.2097203731536865\n",
      "train: step: 56, loss: 1.747999668121338\n",
      "train: step: 57, loss: 1.9586706161499023\n",
      "train: step: 58, loss: 1.5544699430465698\n",
      "train: step: 59, loss: 1.8189806938171387\n",
      "train: step: 60, loss: 1.888069748878479\n",
      "start training model\n",
      "train: step: 61, loss: 1.4362859725952148\n",
      "train: step: 62, loss: 1.6628950834274292\n",
      "train: step: 63, loss: 1.8293648958206177\n",
      "train: step: 64, loss: 1.533456563949585\n",
      "train: step: 65, loss: 1.402876853942871\n",
      "train: step: 66, loss: 1.5375587940216064\n",
      "train: step: 67, loss: 1.6944608688354492\n",
      "train: step: 68, loss: 1.8843793869018555\n",
      "train: step: 69, loss: 1.6433022022247314\n",
      "train: step: 70, loss: 1.6111375093460083\n",
      "train: step: 71, loss: 1.4734020233154297\n",
      "train: step: 72, loss: 1.7539048194885254\n",
      "start training model\n",
      "train: step: 73, loss: 1.468429446220398\n",
      "train: step: 74, loss: 1.6426498889923096\n",
      "train: step: 75, loss: 1.4476886987686157\n",
      "train: step: 76, loss: 1.343079686164856\n",
      "train: step: 77, loss: 1.5054161548614502\n",
      "train: step: 78, loss: 1.5383235216140747\n",
      "train: step: 79, loss: 1.505271315574646\n",
      "train: step: 80, loss: 1.300397515296936\n",
      "train: step: 81, loss: 1.2672178745269775\n",
      "train: step: 82, loss: 1.4293370246887207\n",
      "train: step: 83, loss: 1.2653815746307373\n",
      "train: step: 84, loss: 1.3312283754348755\n",
      "start training model\n",
      "train: step: 85, loss: 1.2519267797470093\n",
      "train: step: 86, loss: 1.179534673690796\n",
      "train: step: 87, loss: 1.0709848403930664\n",
      "train: step: 88, loss: 1.133713960647583\n",
      "train: step: 89, loss: 1.1214673519134521\n",
      "train: step: 90, loss: 1.3284950256347656\n",
      "train: step: 91, loss: 1.4261271953582764\n",
      "train: step: 92, loss: 1.2796128988265991\n",
      "train: step: 93, loss: 1.1269192695617676\n",
      "train: step: 94, loss: 1.3969018459320068\n",
      "train: step: 95, loss: 1.2728428840637207\n",
      "train: step: 96, loss: 1.1605321168899536\n",
      "start training model\n",
      "train: step: 97, loss: 1.0346440076828003\n",
      "train: step: 98, loss: 1.041954517364502\n",
      "train: step: 99, loss: 1.0554324388504028\n",
      "train: step: 100, loss: 0.9245582818984985\n",
      "\n",
      "Evaluation:\n",
      "Saved model checkpoint to model/textCNN/model/my-model-100\n",
      "\n",
      "2020-07-30T20:13:38.378057, step: 100, loss: 1.1907079617182414, acc: 0.75,precision: 0.4108100563457706, recall: 0.44364426617577885, f_beta: 0.4073191115264874sec/batch *\n",
      "train: step: 101, loss: 1.0420544147491455\n",
      "train: step: 102, loss: 1.081252932548523\n",
      "train: step: 103, loss: 1.0601004362106323\n",
      "train: step: 104, loss: 1.0601314306259155\n",
      "train: step: 105, loss: 1.1948682069778442\n",
      "train: step: 106, loss: 1.1874223947525024\n",
      "train: step: 107, loss: 1.014772653579712\n",
      "train: step: 108, loss: 1.1134141683578491\n",
      "start training model\n",
      "train: step: 109, loss: 1.244939923286438\n",
      "train: step: 110, loss: 0.9373803734779358\n",
      "train: step: 111, loss: 0.8531287312507629\n",
      "train: step: 112, loss: 0.9407995343208313\n",
      "train: step: 113, loss: 0.8485614061355591\n",
      "train: step: 114, loss: 1.0707042217254639\n",
      "train: step: 115, loss: 0.9342836141586304\n",
      "train: step: 116, loss: 0.8241806030273438\n",
      "train: step: 117, loss: 0.9053363800048828\n",
      "train: step: 118, loss: 0.7213877439498901\n",
      "train: step: 119, loss: 0.9510024785995483\n",
      "train: step: 120, loss: 1.0058192014694214\n",
      "WARNING:tensorflow:From <ipython-input-4-6a3a68dae750>:183: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info.\n",
      "WARNING:tensorflow:From <ipython-input-4-6a3a68dae750>:192: calling SavedModelBuilder.add_meta_graph_and_variables (from tensorflow.python.saved_model.builder_impl) with legacy_init_op is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Pass your op to the equivalent parameter main_op instead.\n",
      "INFO:tensorflow:No assets to save.\n",
      "INFO:tensorflow:No assets to write.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:SavedModel written to: model/textCNN/savedModel\\saved_model.pb\n"
     ]
    }
   ],
   "source": [
    "layerType = config.layerType\n",
    "# 生成训练集和验证集\n",
    "trainReviews = data.trainReviews\n",
    "trainLabels = data.trainLabels\n",
    "evalReviews = data.evalReviews\n",
    "evalLabels = data.evalLabels\n",
    "wordEmbedding = data.wordEmbedding\n",
    "labelList = data.labelList\n",
    "\n",
    "\n",
    "# 定义计算图\n",
    "with tf.Graph().as_default():\n",
    "    session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)\n",
    "    session_conf.gpu_options.allow_growth=True\n",
    "    session_conf.gpu_options.per_process_gpu_memory_fraction = 0.9  # 配置gpu占用率  \n",
    "    sess = tf.Session(config=session_conf)    \n",
    "    # 定义会话\n",
    "    with sess.as_default():   \n",
    "        if layerType == \"textCNN\":\n",
    "            from layers.textcnn import TextCNN\n",
    "            layer = TextCNN(config, wordEmbedding)\n",
    "        elif layerType == \"textRNN\":\n",
    "            from layers.textrnn import BiLSTM\n",
    "            layer = BiLSTM(config, wordEmbedding)\n",
    "        elif layerType == \"BiLSTMAttention\":\n",
    "            from layers.BiLSTMAttention import BiLSTMAttention\n",
    "            layer = BiLSTMAttention(config, wordEmbedding)\n",
    "        elif layerType == \"textrcnn\":\n",
    "            from layers.textrcnn import RCNN\n",
    "            layer = RCNN(config, wordEmbedding)\n",
    "        elif layerType == \"AdversarialLSTM\":\n",
    "            from layers.AdversarialLSTM import AdversarialLSTM\n",
    "            indexFreqs = data.indexFreqs\n",
    "            layer = AdversarialLSTM(config, wordEmbedding,indexFreqs)\n",
    "        elif layerType == \"Transformer\":\n",
    "            from layers.Transformer import Transformer,fixedPositionEmbedding\n",
    "            embeddedPosition = fixedPositionEmbedding(config.batchSize, config.sequenceLength) \n",
    "            layer = Transformer(config, wordEmbedding)\n",
    "     \n",
    "        globalStep = tf.Variable(0, trainable=False, name='globalStep')\n",
    "        optimizer = tf.train.AdamOptimizer(config.training.learningRate)\n",
    "        gradients, variables = zip(*optimizer.compute_gradients(layer.loss))\n",
    "        # 梯度裁剪\n",
    "        gradients, _ = tf.clip_by_global_norm(gradients, config.training.clip)\n",
    "        # 将梯度应用到变量下，生成训练器\n",
    "        gradsAndVars = zip(gradients, variables)\n",
    "        trainOp = optimizer.apply_gradients(gradsAndVars, global_step=globalStep)\n",
    "         \n",
    "        # 用summary绘制tensorBoard\n",
    "        gradSummaries = []\n",
    "        for g, v in gradsAndVars:\n",
    "            if g is not None:\n",
    "                tf.summary.histogram(\"{}/grad/hist\".format(v.name), g)\n",
    "                tf.summary.scalar(\"{}/grad/sparsity\".format(v.name), tf.nn.zero_fraction(g))\n",
    "        \n",
    "        outDir = os.path.abspath(os.path.join(os.path.curdir, f\"summarys/{layerType}/\"))\n",
    "        print(\"Writing to {}\\n\".format(outDir))        \n",
    "        lossSummary = tf.summary.scalar(\"loss\", layer.loss)\n",
    "        summaryOp = tf.summary.merge_all()        \n",
    "        trainSummaryDir = os.path.join(outDir, \"train\")\n",
    "        trainSummaryWriter = tf.summary.FileWriter(trainSummaryDir, sess.graph)        \n",
    "        evalSummaryDir = os.path.join(outDir, \"eval\")\n",
    "        evalSummaryWriter = tf.summary.FileWriter(evalSummaryDir, sess.graph)       \n",
    "        \n",
    "        # 初始化所有变量\n",
    "        saver = tf.train.Saver(tf.global_variables(), max_to_keep=5)        \n",
    "        # 保存模型的一种方式，保存为pb文件\n",
    "        savedModelPath = f\"model/{layerType}/savedModel\"\n",
    "        if os.path.exists(savedModelPath):\n",
    "            import shutil\n",
    "            shutil.rmtree(savedModelPath)\n",
    "        builder = tf.saved_model.builder.SavedModelBuilder(savedModelPath)            \n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        def trainStep(batchX, batchY):\n",
    "            \"\"\"\n",
    "            训练函数\n",
    "            \"\"\"   \n",
    "            if layerType==\"Transformer\": \n",
    "                feed_dict = {\n",
    "                  layer.inputX: batchX,\n",
    "                  layer.inputY: batchY,\n",
    "                  layer.dropoutKeepProb: config.dropoutKeepProb,\n",
    "                  layer.embeddedPosition: embeddedPosition\n",
    "                }\n",
    "            else:\n",
    "                feed_dict = {\n",
    "                  layer.inputX: batchX,\n",
    "                  layer.inputY: batchY,\n",
    "                  layer.dropoutKeepProb: config.dropoutKeepProb,\n",
    "                }\n",
    "            _, summary, step, loss, predictions = sess.run(\n",
    "                [trainOp, summaryOp, globalStep, layer.loss, layer.predictions],\n",
    "                feed_dict)\n",
    "            timeStr = datetime.datetime.now().isoformat()\n",
    "            \n",
    "#             if config.numClasses == 1:\n",
    "#                 acc, recall, prec, f_beta = get_binary_metrics(pred_y=predictions, true_y=batchY)\n",
    "                \n",
    "#             elif config.numClasses > 1:\n",
    "#                 acc, recall, prec, f_beta = get_multi_metrics(pred_y=predictions, true_y=batchY,labels=labelList)\n",
    "                \n",
    "            trainSummaryWriter.add_summary(summary, step)            \n",
    "            return loss # , acc, prec, recall, f_beta\n",
    "\n",
    "        def devStep(batchX, batchY):\n",
    "            \"\"\"\n",
    "            验证函数\n",
    "            \"\"\"\n",
    "            if layerType==\"Transformer\": \n",
    "                feed_dict = {\n",
    "                  layer.inputX: batchX,\n",
    "                  layer.inputY: batchY,\n",
    "                  layer.dropoutKeepProb: 1.0,\n",
    "                  layer.embeddedPosition: embeddedPosition\n",
    "                }\n",
    "            else:\n",
    "                feed_dict = {\n",
    "                  layer.inputX: batchX,\n",
    "                  layer.inputY: batchY,\n",
    "                  layer.dropoutKeepProb: 1.0\n",
    "                }\n",
    "            summary, step, loss, predictions = sess.run(\n",
    "                [summaryOp, globalStep, layer.loss, layer.predictions],\n",
    "                feed_dict)\n",
    "            \n",
    "            if config.numClasses == 1:            \n",
    "                acc, precision, recall, f_beta = get_binary_metrics(pred_y=predictions, true_y=batchY)\n",
    "            elif config.numClasses > 1:\n",
    "                acc, precision, recall, f_beta = get_multi_metrics(pred_y=predictions, true_y=batchY, labels=labelList)            \n",
    "            evalSummaryWriter.add_summary(summary, step)\n",
    "            return loss, acc, precision, recall, f_beta\n",
    "        \n",
    "        bestF1Score = 0     # 验证集的最优 F1-score\n",
    "        lastImproved = 0  # record global_step at best_val_accuracy\n",
    "        requireImprovement = 1000  # break training if not having improvement over 1000 iter\n",
    "        flag=False\n",
    "        for i in range(config.training.epoches):\n",
    "            # 训练模型\n",
    "            print(\"start training model\")\n",
    "            for batchTrain in nextBatch(trainReviews, trainLabels, config.batchSize):\n",
    "#                 loss, acc, prec, recall, f_beta = trainStep(batchTrain[0], batchTrain[1])\n",
    "                loss = trainStep(batchTrain[0], batchTrain[1])\n",
    "                currentStep = tf.train.global_step(sess, globalStep) \n",
    "                print(\"train: step: {}, loss: {}\".format(\n",
    "                    currentStep, loss))\n",
    "                if currentStep % config.training.evaluateEvery == 0:\n",
    "                    print(\"\\nEvaluation:\")\n",
    "                    losses = []\n",
    "                    accs = []\n",
    "                    f_betas = []\n",
    "                    precisions = []\n",
    "                    recalls = []\n",
    "                    for batchEval in nextBatch(evalReviews, evalLabels, config.batchSize):\n",
    "                        loss, acc, precision, recall, f_beta = devStep(batchEval[0], batchEval[1])\n",
    "                        losses.append(loss)\n",
    "                        accs.append(acc)\n",
    "                        f_betas.append(f_beta)\n",
    "                        precisions.append(precision)\n",
    "                        recalls.append(recall)\n",
    "                           \n",
    "                    improvedStr = ''\n",
    "                    if bestF1Score<f_beta:\n",
    "                        # 保存模型的另一种方法，保存checkpoint文件\n",
    "                        bestF1Score=f_beta\n",
    "                        improvedStr = '*'\n",
    "                        lastImproved=currentStep\n",
    "                        path = saver.save(sess, f\"model/{layerType}/model/my-model\", global_step=currentStep)\n",
    "                        print(\"Saved model checkpoint to {}\\n\".format(path))\n",
    "                    \n",
    "                    timeStr = datetime.datetime.now().isoformat()\n",
    "                    print(\"{}, step: {}, loss: {}, acc: {},precision: {}, recall: {}, f_beta: {}sec/batch {}\".format(timeStr, currentStep, mean(losses), \n",
    "                                                                                                       mean(accs), mean(precisions),\n",
    "                                                                                                        mean(recalls), mean(f_betas),improvedStr))\n",
    "                    \n",
    "                    if currentStep - lastImproved > requireImprovement:\n",
    "                        print(\"No optimization over 1000 steps, stop training\")\n",
    "                        flag = True\n",
    "                        break\n",
    "            if flag:\n",
    "                break\n",
    "            config.training.learningRate *= config.training.lrDecay\n",
    "                    \n",
    "        inputs = {\"inputX\": tf.saved_model.utils.build_tensor_info(layer.inputX),\n",
    "                  \"keepProb\": tf.saved_model.utils.build_tensor_info(layer.dropoutKeepProb)}\n",
    "\n",
    "        outputs = {\"predictions\": tf.saved_model.utils.build_tensor_info(layer.predictions)}\n",
    "\n",
    "        prediction_signature = tf.saved_model.signature_def_utils.build_signature_def(inputs=inputs, outputs=outputs,\n",
    "                                                                                      method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME)\n",
    "        legacy_init_op = tf.group(tf.tables_initializer(), name=\"legacy_init_op\")\n",
    "        builder.add_meta_graph_and_variables(sess, [tf.saved_model.tag_constants.SERVING],\n",
    "                                            signature_def_map={\"predict\": prediction_signature}, legacy_init_op=legacy_init_op)\n",
    "\n",
    "        builder.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
